# coding=utf-8
import sys
import time
import requests
import pandas as pd
import datetime

root_url = 'http://www.weather.com.cn/'
city_json_url = 'https://j.i8tq.com/weather2020/search/city.js'
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36"
}


def weather40d(areaid):
    '''
    根据areaid爬取40天预报
    :param areaid:
    :return: 返回list类型  或者None
    '''
    t_stamp = int(time.time() * 1000)
    month = str(datetime.datetime.now().year) + str(datetime.datetime.now().month).zfill(2)

    url = 'http://d1.weather.com.cn/calendar_new/2021/{0}_{1}.html?_={2}'.format(areaid, month, t_stamp)
    print(url)

    response = requests.get(url, headers=headers)  # 生成一个response对象
    print("状态码:" + str(response.status_code))  # 打印状态码
    if response.status_code != 200:
        return None

    res = list()  # 保存结果
    s = response.text[response.text.find('['):]
    obj = eval(s)

    for ele in obj:
        res.append({'date': ele['date'], 'max_temp': ele['hmax'], 'min_temp': ele['hmin'], 'rain_poss': ele['hgl'][0:-2]})
    return res


if __name__ == '__main__':

    # step1: 获取城市信息，得到AREAID
    response = requests.get(city_json_url, headers=headers)
    print("状态码:" + str(response.status_code))  # 打印状态码
    if response.status_code != 200:
        print("获取城市信息失败")
        sys.exit()

    # step2: 遍历城市信息
    full = pd.DataFrame()
    citys = eval(response.text[response.text.find('{'):])
    for province in citys:
        province_name = province
        # full[province] = dict()
        province = citys[province]

        for city in province:
            city_name = city
            # full[province_name][city_name] = list()
            city = province[city]

            for area in city:
                try:
                    areaid = city[area]['AREAID']
                    name = city[area]['NAMECN']

                    # full[province_name][city_name].append({name+areaid:weather40d.weather40d(areaid)})
                    # 根据AREAID获取40天和1天的天气预报
                    base = pd.DataFrame.from_dict(weather40d(areaid))
                    if len(base) == 0:
                        print("该网页有问题")
                        continue
                    # 补充列信息
                    base['area'] = name
                    base['city'] = city_name
                    base['province_name'] = province_name
                    # 保存至dataframe
                    full = full.append(base)
                    print('采集' + name + "完毕")
                    time.sleep(2)
                except:
                    continue

    # df = pd.DataFrame.from_dict(full['北京']['北京'])

    # 保存
    full.to_csv('./weather40d.csv')
    full.to_csv('./weather40d_without_index.csv', index=None)
